
package models.spider;

import models.spider.index.Entry;
import models.spider.index.Index;
import models.spider.index.Queue;
import models.spider.request.Factory;
import models.spider.request.Request;
import models.spider.sink.ISink;

import org.apache.commons.lang3.StringUtils;

import play.Logger;

/**
 * 
 * 
 * @author laurent@opprecht.info
 */
public class Spider implements ISpider {

    /**
     * Empty spider, does nothing
     * 
     * @author laurent
     * 
     */
    public static class Empty extends Spider {

        private Empty() {
            super(models.spider.config.Factory.empty);
        }

        @Override
        public boolean crawl(final int count) {
            return false;
        }

        @Override
        public boolean crawl() {
            return false;
        }

        @Override
        public void add(final String url) {
        }

        @Override
        public void refresh(final String url) {
            ;
        }

    }

    public static final Spider empty = new Empty();

    protected Index            index;
    protected Queue            queue;
    protected ISink            sink;
    protected Factory          requestFactory;

    public Spider(final models.spider.config.Factory config) {
        this.queue = config.newQueue();
        this.index = config.newIndex();
        this.sink = config.newSink();
        this.requestFactory = config.newRequestFactory();
    }

    public boolean crawl(final int count) {
        for (int i = 0; i < count; i++) {
            boolean success = crawl();
            if (!success) {
                return false;
            }
        }
        return true;
    }

    public boolean crawl() {
        Entry entry = queue.remove();
        if (entry == null) {
            return false;
        }

        String url = entry.getUrl();
        if (StringUtils.isEmpty(url)) {
            return false;
        }

        // List<String> links = new ArrayList<>();
        Request request = requestFactory.create(url);
        try {
            // must think invalid request as well as valid one
            sink.sink(request, this);
        } catch (Exception ex) {
            Logger.debug("spider crawl error ", ex);
        } finally {
            if (request.isValid()) {
                index.success(url);
                // for (String link : links) {
                // add(link);
                // }
            } else {
                index.failure(url);
            }
        }
        return true;
    }

    @Override
    public void add(final String url) {
        queue.add(url);
    }

    @Override
    public void refresh(final String url) {
        queue.refresh(url);
    }
}
